import numpy as np
import os
import csv
import json
collist=['wip', 'haventseen', 'ootd', 'bekind', 'personalfinance', 'cozyathome', 'RoomTour', 'theatrekids', 'ImAGhost', 'holidaytiktok', 'halloweenlook', 'happyhalloween', 'welldone', 'motivationmonday', 'thinkingabout', 'nonuancenovember', 'ourtype', 'fanedit', 'needtoknow', 'cleantok', 'graphicdesign', 'readysetshop', 'holidaysourway', 'onlinedating', 'myhobby', 'tiktokfood', 'whereilive', 'myrecommendation', 'worldseries', 'animation', 'cocinando', 'easydiy', 'diceroll', 'rnbvibes', 'festivefashion', 'holidaydecor', 'nbadraft', 'halloweenishere', 'christmas2020', 'howbizarre', 'sfxmakeup', 'givingthanks', 'holidayvibes', 'gamenight', 'happyholidays', 'carsoftiktok', 'fallguysmoments', 'interiordesign', 'homecooked', 'veteransday', 'youwantmore', 'coldweather', 'wildanimals', 'mycostume', 'meleaving', 'mypfp', 'catchphrases', 'watchmegrow', 'holidaycrafts', 'growupwithme', 'clingypet', 'happyhanukkah', 'lunarnewyear', 'tabletop', 'comfortfood', 'selfimprovement', '2021affirmations', 'perfectmatch', 'givingszn', 'holidaycountdown', 'bakingszn', 'holidaymusic', 'familyimpression', 'inkdrawing', 'WeekendVibes', 'recordsday', 'productivity', 'smallbusiness', 'falldiy', 'whenwewereyounger', 'yellow', 'ComingOfAge', 'artmas', 'gaminglife', 'gamingsetup', 'hellowinter', 'planttiktok', 'housetour', 'neonshadow', 'homeoffice', 'raisedby', 'makeitvogue', 'foodtiktok', 'valentinesday', 'yougotthis', 'stemlife']
ids={}
for col in collist:
    ids[col]={}
    dtc=0
    for file in os.listdir('D:\\Work\\Tool\\tiktok\\TikToks\\'):
        if col in file:
            dt = file.split('_')[2].replace('.json', '')
            if dtc >=14 :
                break
            dtc+=1
            ids[col][dt]=[]

            with open('D:\\Work\\Tool\\tiktok\\TikToks\\' + file, 'r', encoding='utf-8',
                      newline='\n') as filename_input:
                lc = 0
                for line in filename_input:
                    z = json.loads(line)
                    if lc >= 100:
                        break
                    k = ''
                    if 'id' in z.keys():
                        k = z['id']
                    elif 'itemInfos' in z.keys():
                        k = z['itemInfos']['id']
                    if k not in ids[col][dt]:
                        ids[col][dt].append(k)
                    lc += 1
with open('D:\\Work\\kusuri\\newlyaddedids.tsv','w',newline='\n',encoding='utf-8')as fin:
    writer = csv.writer(fin, delimiter='\t')
    for col in collist:
        prev=None
        for dt in ids[col].keys():
            if prev is None:
                writer.writerow([col,dt])
                prev=dt
                continue
            temp=[col,dt]
            temp.extend(list(set(ids[col][dt])-set(ids[col][prev])))
            prev=dt
            writer.writerow(temp)